import sys, itertools
import getopt

class ScanError(SystemError): pass

__MD5OPTIONSTR = "# md5"

def verify_checksum_line_in_prepfile(optionLine, md5str):
    if optionLine.startswith(__MD5OPTIONSTR):
        s = optionLine[len(__MD5OPTIONSTR):].strip()
        if s != md5str:
            return False
    return True

def scan_clone_data_file(dataFile, commentSink, fileSink, cloneSink, 
        fileBegin=None, fileEnd=None, cloneBegin=None, cloneEnd=None):
    def scan_file_line(lineIndex, line):
        fields = line.split("\t")
        assert len(fields) >= 2
        fileIDStr, pathStr = fields[:2]
        fileID = int(fileIDStr)
        lenStr = None if len(fields) < 3 or fields[2] == '-' else int(fields[2])
        checksumStr = None if len(fields) < 4 or fields[3] == '-' else fields[3]
        if lenStr == '-':
            fileSink(fileID, pathStr, None, None)
        else:
            length = int(lenStr)
            fileSink(fileID, pathStr, length, checksumStr)
    
    cloneClass = []
    def scan_clone_line(lineIndex, line):
        if line == "{":
            cloneClass[:] = [[]]
        elif line == "--":
            cloneClass.append([])
        elif line == "}":
            if cloneClass:
                cloneSink(cloneClass)
        else:
            fileIDStr, description = line.split("\t")
            fileID = int(fileIDStr)
            r = []
            for d in description.split(","):
                startPosStr, lenStr = d.split("+")
                r.append(( int(startPosStr), int(lenStr) ))
            cloneClass[-1].append(( fileID, r ))
    
    def scan_error(lineIndex, line):
        raise ScanError("invalid line (%d)" % (lineIndex + 1))
    
    def nop(): pass
    if not fileBegin: fileBegin = nop
    if not fileEnd: fileEnd = nop
    if not cloneBegin: cloneBegin = nop
    if not cloneEnd: cloneEnd = nop
    
    scanLineFunc = scan_error
    for lineIndex, line in enumerate(dataFile):
        line = line.rstrip()
        if line and line[0] == '#':
            commentSink(line)
        elif line.endswith(":"):
            if line == "files:":
                scanLineFunc = scan_file_line
                fileBegin()
            elif line == "clone classes:":
                fileEnd()
                scanLineFunc = scan_clone_line
                cloneBegin()
            else:
                raise ScanError("invaild line (%d)" % (lineIndex + 1))
        else:
            scanLineFunc(lineIndex, line)
    else:
        cloneEnd()

def clone_pretty(cloneClass):
    r = []
    r.append("{")
    for subSet in cloneClass:
        for fileID, ranges in subSet:
            r.append("%d\t%s" % ( fileID, ",".join("%d+%d" % range for range in ranges) ))
        r.append("--")
    r.pop() # remove the last "--"
    r.append("}")
    return r

def __extract_option(argv):
    optionCounting = None
    diagonals = None
    nondiagonals = None
    
    opts, args = getopt.gnu_getopt(argv, "cd:n:")
    for k, v in opts:
        if k == "-c":
            optionCounting = True
        elif k == "-d":
            assert v in ("+", "-", "on", "off")
            diagonals = v in ("+", "on")
        elif k == "-n":
            assert v in ("+", "-", "on", "off")
            nondiagonals = v in ("+", "on")
    if diagonals == False and nondiagonals == False:
        raise SystemError('both "-d -" and "-n -" are specified')
    
    return ( optionCounting, diagonals, nondiagonals ), args

def __do_pickup(argv):
    ( optionCounting, diagonals, nondiagonals ), argv = __extract_option(argv)
    if diagonals is not None or nondiagonals is not None:
        raise SystemError("option -d, -n is not applicable in pickup mode")
    
    dataFile, fileIDStrs = argv[0], argv[1:]
    targetFileIDs = map(int, fileIDStrs)
    targetFileIDSet = set(targetFileIDs)
    
    def nop(*a): pass
    
    class CloneSink(object):
        def __call__(self, cloneClass):
            fileIDSet = set()
            for subSet in cloneClass:
                for item in subSet:
                    fileIDSet.add(item[0])
            if not (targetFileIDSet - fileIDSet): # targetFileIDSet is covered by fileIDSet
                self.found(cloneClass)
        def found(self, cloneClass): pass
        def done(self): pass
    
    if optionCounting:
        class CloneCount(CloneSink):
            def __init__(self): self.count = 0
            def found(self, cloneClass): self.count += 1
            def done(self): print "%d" % self.count
        cloneSink = CloneCount()
    else:
        class ClonePrint(CloneSink):
            def found(self, cloneClass): print "\n".join(clone_pretty(cloneClass))
        cloneSink = ClonePrint()
    
    f = open(dataFile, "r")
    scan_clone_data_file(f, nop, nop, cloneSink)
    cloneSink.done()
    f.close()

def is_diagonal(fileIDPair):
    f1, f2 = fileIDPair
    return f1 == f2

def is_nondiagonal(fileIDPair):
    f1, f2 = fileIDPair
    return f1 != f2

def __do_sub(argv):
    ( optionCounting, diagonals, nondiagonals ), argv = __extract_option(argv)
    if diagonals is None: diagonals = True # default value
    if nondiagonals is None: nondiagonals = True # default value
    
    dataFiles = []
    
    for a in argv:
        dataFiles.append(a)
    
#    def nop1(a1): pass
#    commentSink = nop1
#    def nop2(a1, a2): pass
#    fileSink = nop2
#    def cloneSink(cloneClass):
#        print repr(cloneClass)
#    
#    for df in dataFiles:
#        f = open(df, "r")
#        scan_clone_data_file(f, commentSink, fileSink, cloneSink)
#        f.close()
    
    if len(dataFiles) < 2:
        SystemError, "need at least two data files"
    
    def nop(*a): pass
    
    fileTable = []
    def fileSink(fileID, filePath, length, checksum):
        fileTable.append(( fileID, filePath ))
    cloneCoveringFilePairSet = set()
    def cloneSink(cloneClass):
        fileIDSet = set()
        fileIDMultipleAppearingSet = set()
        for subSet in cloneClass:
            for item in subSet:
                fileID = item[0]
                if fileID in fileIDSet:
                    fileIDMultipleAppearingSet.add(fileID)
                else:
                    fileIDSet.add(fileID)
        fileIDs = sorted(fileIDSet)
        for i, f1 in enumerate(fileIDs):
            cloneCoveringFilePairSet.update(( f1, f2 ) for f2 in fileIDs[i+1:])
        for f in fileIDMultipleAppearingSet:
            cloneCoveringFilePairSet.add(( f, f ))
    
    fileTableAndCovers = []
    for df in dataFiles:
        f = open(df, "r")
        scan_clone_data_file(f, nop, fileSink, cloneSink)
        f.close()
        fileTableAndCovers.append(( sorted(fileTable), sorted(cloneCoveringFilePairSet) ))
        fileTable[:] = []
        cloneCoveringFilePairSet.clear()
    
    for i, ( ftac1, ftac2 ) in enumerate(zip(fileTableAndCovers, fileTableAndCovers[1:])):
        if ftac1[0] != ftac2[0]:
            raise SystemError("file set of data %d doesn't equal to ones of %d" % ( i, i + 1 ))
    
    s = set(fileTableAndCovers[0][1])
    for ftac in fileTableAndCovers[1:]:
        s.difference_update(ftac[1])
    coverDifference = sorted(s)
    
    if not diagonals:
        coverDifference = filter(is_nondiagonal, coverDifference)
    elif not nondiagonals:
        coverDifference = filter(is_diagonal, coverDifference)
        
    print "# subtract: %s" % ", ".join(dataFiles)
    if optionCounting:
        count = 0
        for f1, g in itertools.groupby(coverDifference, lambda p: p[0]):
            count += len(list(g))
        print "%d" % count
    else:
        for f1, g in itertools.groupby(coverDifference, lambda p: p[0]):
            print "%d\t%s" % ( f1, ",".join("%d" % p[1] for p in g) )

class CoverExtractCloneSink(object):
    def __init__(self):
        self.cloneCoveringFilePairSet = set()
    def __call__(self, cloneClass):
        fileIDMultipleAppearingSet = set()
        fileIDSet = set()
        for subSet in cloneClass:
            for item in subSet:
                fileID = item[0]
                if fileID in fileIDSet:
                    fileIDMultipleAppearingSet.add(fileID)
                else:
                    fileIDSet.add(fileID)
        fileIDs = sorted(fileIDSet)
        for i, f1 in enumerate(fileIDs):
            self.cloneCoveringFilePairSet.update(( f1, f2 ) for f2 in fileIDs[i+1:])
        self.cloneCoveringFilePairSet.update(( f, f ) for f in fileIDMultipleAppearingSet)

def __do_cover(argv):
    ( optionCounting, diagonals, nondiagonals ), argv = __extract_option(argv)
    if diagonals is None: diagonals = True # default value
    if nondiagonals is None: nondiagonals = True # default value
    
    if not argv:
        raise SystemError("no file is given")
    elif len(argv) >= 2:
        raise SystemError("too many files")
    
    dataFile = argv[0]
    
    def nop(*a): pass
    
    fileTable = []
    def fileSink(fileID, filePath, length, checksum):
        fileTable.append(( fileID, filePath ))
    coverExtractCloneSink = CoverExtractCloneSink()
    f = open(dataFile, "r")
    scan_clone_data_file(f, nop, fileSink, coverExtractCloneSink)
    f.close()
    
    cloneCoveringFilePairs = sorted(coverExtractCloneSink.cloneCoveringFilePairSet)
    if not diagonals:
        cloneCoveringFilePairs = filter(is_nondiagonal, cloneCoveringFilePairs)
    elif not nondiagonals:
        cloneCoveringFilePairs = filter(is_diagonal, cloneCoveringFilePairs)
        
    print "# cover: %s" % dataFile
    if optionCounting:
        count = 0
        for f1, g in itertools.groupby(cloneCoveringFilePairs, lambda p: p[0]):
            count += len(list(g))
        print "%d" % count
    else:
        for f1, g in itertools.groupby(cloneCoveringFilePairs, lambda p: p[0]):
            print "%d\t%s" % ( f1, ",".join("%d" % p[1] for p in g) )
    
if __name__ == '__main__':
    usage = """
usage 1: cldt_datamanip sub OPTIONS left right1 right2 ...
  Subtract. Finds file pairs which include some code fragments of left
  but don't include ones from right1 or right2 ...
usage 2: cldt_datamanip pick OPTIONS clone_data file_id_1 file_id_2 ...
  Pick up. Finds clone sets which includes all of the files.
usage 3: cldt_datamanip cover OPTIONS clone_data
  Cover. Finds file pairs which include some code fragments of clone_data.
option
  -c: counts. prints count of file pairs or clone sets.
  -d +, -d -: turns on/off diagonal clones. diagonal means that
      code fragments of a clone pair coming from one file. (-d +)
  -n +, -n -: turns on/off non-diagonal clones. non-diagonal means that
      code fragments of a clone pair coming from distinct files.
"""

    if len(sys.argv) == 1 or sys.argv[1:] == [ "-h" ]:
        print usage
        sys.exit(0)
    
    argv1 = sys.argv[1]
    if argv1 in ("sub", "subtract" ):
        __do_sub(sys.argv[2:])
    elif argv1 == "pick":
        __do_pickup(sys.argv[2:])
    elif argv1 == "cover":
        __do_cover(sys.argv[2:])
    else:
        raise SystemError("unknown command")
